# import requests
import time
import threading
# from bs4 import BeautifulSoup
# from queue import Queue
# import re
# import os
# from urllib.request import urlretrieve


gLock=threading.Lock()
times=0
maxThtreadNum=5000000
checkInterval=1


#用于检测爬虫状态
def chect(keys):
    global times
    global checkInterval

    allKeys=len(keys)
    preTime=time.time()
    preTimes=times
    speed=0
    surplusTime='-'
    while True:
        if time.time()-preTime==0:
            speed=0
        else:
            speed=(times-preTimes)/(time.time()-preTime)
        if speed==0:
            surplusTime='-'
        else:
            surplusTime=str(int((allKeys-times)/speed))
        print('当前线程数%d,%d%%已完成,爬取速度%d次/秒，预计还需%s秒'%(len(threading.enumerate()),int((times/allKeys)*100),speed,surplusTime))
        preTime=time.time()
        preTimes=times
        time.sleep(checkInterval)
        if times==len(keys):
            return


#得到key列表
def getKeys():
    keys=[1]*10
    return keys


#爬虫线程
def spider(key):
    #使用前必须声明全局变量
    global gLock
    global times

    #以下用来数据提取
    time.sleep(1)
    
    #以下用来数据存储
    gLock.acquire()
    
    #times用来计数
    times+=1
    gLock.release()


#调配爬虫线程的函数
def spiderThread(keys):
    global gLock
    global maxThtreadNum

    #打开检测爬虫进度线程
    t=threading.Thread(target=chect,args=(keys,))
    t.start()

    #打开所有爬虫线程
    for key in keys:
        #这是单线程爬虫的写法
        # spider(key)

        #这是多线程爬虫的写法
        while True:
            threadNum=len(threading.enumerate())#得到目前有多少线程
            if threadNum<=maxThtreadNum:
                t=threading.Thread(target=spider,args=(key,))
                t.start()
                break

    #判断是否完成爬取            
    while True:
        if times==len(keys):
            return
        # print(str(int((times/len(keys))*100))+'%已完成')
        

if __name__ == "__main__":
    startTime=time.time()
    spiderThread(getKeys())

    #这里对爬取到的数据进行处理


    print('共用了%f秒'%(time.time()-startTime))


    